#!/usr/bin/python2.4
#
# Copyright 2011 Google Inc. All Rights Reserved.

"""Automated scraper for Energex Qld's power interruption data.

This module implements a web request handler that will update a memcache
variable with a KML representing Energex Qld's current power outage data. Upon
request, it will also serve this information from the memcache.
"""

__author__ = 'bst@google.com (Ben Stewart)'

import datetime
import logging
import urllib

from BeautifulSoup import BeautifulSoup as BS
import controllers
from controllers import geocode
from django.utils import simplejson, html

# Region IDs, defined at Energex's side.
ENERGEX_REGIONS = [42, 43, 44, 45, 47, 49, 50, 51, 53, 55, 56, 57, 58]

# Number of result pages to scrape per region.
# TODO(bst): Up this range if we are presented with a lot of power outages
ENERGEX_PAGES_PER_REGION = 2


def fixBr(o):
  """Convert a string or a HTML tag to a string.

  Args:
    o: Either a string, or a HTML element.

  Returns:
    If o is a string, returns o.
    If o is a <br> element, returns a newline.
    If o is anything else, return an empty string.
  """

  if isinstance(o, basestring):
    return o
  elif o.name == 'br':
    return "\n"
  else:
    return ''


def fixBrList(l):
    """Convert a list of strings and HTML elements to a string.

    Args:
      l: Iterable with strings and/or HTML elements.

    Returns:
      A string of all the strings concatenated, after elements have been
      transformed by fixBr().
    """

    return reduce(lambda a, b: a + b, map(fixBr, l))


def parseEnergexFlood(content):
  suburbs = {}
  page = BS(content)

  for div in page.findAll('div', id='networksub'):
    for table in filter(lambda elem: not isinstance(elem, basestring) and
                        elem.name == 'table', div.contents):
        for row in filter(lambda r: not isinstance(r, basestring),
                          table.contents)[1:]:

          cells = filter(lambda r: not isinstance(r, basestring), row)

          if len(cells) >= 4:
            if cells[0].name == 'td':
              data = {
                  'street': fixBrList(cells[1].contents),
                  'suburb': fixBrList(cells[0].contents),
                  'state': 'Queensland',
                  'country': 'Australia',
                  'cause': 'Affected by Flooding',
                  'etr': fixBrList(cells[2].contents),
                  'updated': fixBrList(cells[3].contents),
                  }

              suburbs['%s, %s, %s' %
                      (data['street'], data['suburb'], data['state'])] = data
              logging.debug('energex: Added street %s, %s, %s', data['street'],
                            data['suburb'], data['state'])

  return suburbs


def scrapeEnergexSite():
  """Scrape the Energex site for supply interruption information.

  We scrape the Energex site by region to obtain a list of areas that are
  experiencing power interruptions. This is done by region ID, a list of which
  are given in ENERGEX_REGIONS. Those IDs were found on the Energex site, but
  may be an incomplete list - we should check from time to time whether these
  are up to date. Or just make a smarter scraper.

  We fetch each region's data, and look for a table with width=95%. If one is
  found, we iterate across the rows, then the cells to find suburb, cause and
  resolution time.

  While running the scrape, we store the data in a dictionary, keyed by
  locality. This is to avoid duplicate entries - the Energex list appears to
  include multiple reports for some areas, added at regular intervals. We take
  the most recent one, which to this time, has been the last one on the list.

  Returns:
    An array of dictionaries, with the following keys:

    - suburb: Name of the suburb
    - state: QLD
    - country: Australia
    - cause: Reason for the fault
    - etr: Estimated time of restoration
  """
  suburbs = {}

  if False:
    for region in ENERGEX_REGIONS:
      for pageno in xrange(1,1+ENERGEX_PAGES_PER_REGION):
        url = ('http://www.energex.com.au/network/asp/unplanned_interruptions' +
               '.asp?cmd=regiondetail&id=%d&index=%d') % (region, pageno)
        logging.info('Fetching URL %s', url)
        raw = urllib.urlopen(url)

        suburbs.update(parseEnergex(raw.read()))

  url = 'http://www.energex.com.au/qldfloods/flood_power_restoration.html'
  logging.info('Fetching URL %s', url)
  raw = urllib.urlopen(url)

  suburbs.update(parseEnergexFlood(raw.read()))

  print suburbs
  return suburbs.values()


def getMap():
  """Generate KML for current map outages.

  This routine will fetch the current outage information from Energex and hand
  it off to makeKml(), which will do all the geocoding and assembly required to
  generate a KML document.

  Returns:
    KML document as a string.
  """

  suburbs = []
  suburbs.extend(scrapeEnergexSite())
  # TODO(bst): Add other providers' data here!

  logging.info('electricity: Found %d suburbs', len(suburbs))
  return makeKml(suburbs)


# HERE BE DRAGONS
# TODO(jmcgill): Replace with Fusion Tables when we have time to play with OAuth
# and verify that upload/drop is reliable.
def makeStyle(document, image, id):
  """Generate a style definition with a given icon.

  Args:
    document:  Current KML document
    image:  Icon image URL
    id:  XML ID for the style, without the # prefix.
  """

  text = """<Style id="%s">
    <IconStyle>
      <Icon>
        <href>%s</href>
      </Icon>
    </IconStyle>
  </Style>
  """ % (id, image)
  document.append(text)


def makePoint(document, point, style = '#powerOutage'):
  """Generate a placemark with the given data and style.

  Args:
    document:  Current KML document
    point:  Outage information; dict
    style:  Style URL

  Returns:
    Nothing.
  """

  streetAddr = filter(lambda a: a != '', map(lambda k: point[k], ['street',
                                                                  'suburb']))
  streetAddr = ', '.join(streetAddr + ['Australia'])


  lat, lng, approx = geocode.GetLocation(point['street'] + ', ' + point['suburb'],
                                 point['state'], flip_address=False, flush=True)
  #TODO(bst): Check for ]]> in embedded html

  text = """<Placemark>
    <styleUrl>%s</styleUrl>
    <name>%s</name>
    <description>
      <![CDATA[
        <p><b>Electricity Outage: %s</b></p>
        <p><b>Cause:</b> %s</p>
        <p><b>Restoration Time:</b> %s (may vary up to 12h due to storm activity)</p>""" % (style,
      html.escape(', '.join([point['street'],point['suburb']])),
      html.escape(', '.join([point['street'],point['suburb'],point['state']])),
      html.escape(point['cause']),
      html.escape(point['etr']))


  if 'updated' in point:
    text += "<p><b>Last Updated:</b> %s</p>" % html.escape(point['updated'])

  text += """
      ]]>
    </description>
    <Point>
      <coordinates>%s,%s</coordinates>
    </Point>
  </Placemark>""" % (lat, lng)

  document.append(text)


def makeKml(data):
  # Why bother with the DOM when you can concat strings?
  # TODO(bst): sanitise the KML
  document = []
  document.append("""<?xml version="1.0" encoding="utf-8"?>
  <kml xmlns="http://earth.google.com/kml/2.0">
  <Document><name>Queensland Power Outages</name>""")

  # Create icon styles.
  base = "http://mapvisage.appspot.com/static/floodmap/images/rivers/"
  makeStyle(document, base + "red_smaller.png", "red")
  makeStyle(document, base + "green_smaller.png", "green")
  makeStyle(document, base + "orange_smaller.png", "orange")
  makeStyle(document, base + "ltblue_smaller.png", "ltblue")
  makeStyle(document, base + "gray_smaller.png", "gray")
  makeStyle(document, 'http://mapvisage.appspot.com/static/floodmap/images/' +
            'power-outage.png', 'powerOutage')

  # Add points - TODO(jmcgill) Infowindows?
  for point in data:
    makePoint(document, point)

  document.append("</Document></kml>")
  return "".join(document)


def updateMap():
  from google.appengine.api import memcache
  # TODO(arb): also store in datastore
  memcache.set('qldpoweroutages_new', getMap())


def showMap():
  from google.appengine.api import memcache
  return memcache.get('qldpoweroutages_new')

from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app


class UpdatePage(webapp.RequestHandler):
  """Request handler to update the electricity outage KML."""

  def get(self):
    """Serve a request to update the map.

    On success, "OK" will be returned to the browser.
    """

    updateMap()
    self.response.headers['Content-Type'] = 'text/plain'
    self.response.out.write('OK')


class ShowPage(webapp.RequestHandler):
  """Request handler to serve a KML file."""

  def get(self):
    """Serve the generated KML to the user.

    We present the KML to the user with the content type
    application/vnd.google-earth.kml+xml. If we want to serve the KML zipped, we
    will have to change this.
    """

    controllers.cached_response(self.response, duration=300)
    self.response.headers['Content-Type'] = 'application/vnd.google-earth.kml+xml'
    self.response.out.write(showMap())


application = webapp.WSGIApplication([('/kml/qldpoweroutages/update__', UpdatePage),
                                      ('/kml/qldpoweroutages', ShowPage),
                                      ('/electricity/update__', UpdatePage),
                                      ('/electricity/json9', ShowPage)], debug=True)

def main():
  run_wsgi_app(application)

if __name__ == "__main__":
  main()
